


<!DOCTYPE html>
<!--[if IE 8]><html class="no-js lt-ie9" lang="en" > <![endif]-->
<!--[if gt IE 8]><!--> <html class="no-js" lang="en" > <!--<![endif]-->
<head>
  <meta charset="utf-8">
  
  <meta name="viewport" content="width=device-width, initial-scale=1.0">
  
  <title>torch.distributed.rpc.api &mdash; PyTorch master documentation</title>
  

  
  
  
  
    <link rel="canonical" href="https://pytorch.org/docs/stable/_modules/torch/distributed/rpc/api.html"/>
  

  

  
  
    

  

  <link rel="stylesheet" href="../../../../_static/css/theme.css" type="text/css" />
  <!-- <link rel="stylesheet" href="../../../../_static/pygments.css" type="text/css" /> -->
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.10.0-beta/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/css/jit.css" type="text/css" />
  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/katex@0.11.1/dist/katex.min.css" type="text/css" />
  <link rel="stylesheet" href="../../../../_static/katex-math.css" type="text/css" />
    <link rel="index" title="Index" href="../../../../genindex.html" />
    <link rel="search" title="Search" href="../../../../search.html" /> 

  
  <script src="../../../../_static/js/modernizr.min.js"></script>

  <!-- Preload the theme fonts -->

<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-book.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/IBMPlexMono/IBMPlexMono-Medium.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/FreightSans/freight-sans-medium-italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="../../../../_static/fonts/IBMPlexMono/IBMPlexMono-SemiBold.woff2" as="font" type="font/woff2" crossorigin="anonymous">

<!-- Preload the katex fonts -->

<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Math-Italic.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Main-Bold.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size1-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size4-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size2-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Size3-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
<link rel="preload" href="https://cdn.jsdelivr.net/npm/katex@0.10.0/dist/fonts/KaTeX_Caligraphic-Regular.woff2" as="font" type="font/woff2" crossorigin="anonymous">
</head>

<div class="container-fluid header-holder tutorials-header" id="header-holder">
  <div class="container">
    <div class="header-container">
      <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>

      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <div class="ecosystem-dropdown">
              <a id="dropdownMenuButton" data-toggle="ecosystem-dropdown">
                Ecosystem
              </a>
              <div class="ecosystem-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/hub"">
                  <span class=dropdown-title>Models (Beta)</span>
                  <p>Discover, publish, and reuse pre-trained models</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/ecosystem">
                  <span class=dropdown-title>Tools & Libraries</span>
                  <p>Explore the ecosystem of tools and libraries</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <div class="resources-dropdown">
              <a id="resourcesDropdownButton" data-toggle="resources-dropdown">
                Resources
              </a>
              <div class="resources-dropdown-menu">
                <a class="nav-dropdown-item" href="https://pytorch.org/resources"">
                  <span class=dropdown-title>Developer Resources</span>
                  <p>Find resources and get questions answered</p>
                </a>
                <a class="nav-dropdown-item" href="https://pytorch.org/features">
                  <span class=dropdown-title>About</span>
                  <p>Learn about PyTorch’s features and capabilities</p>
                </a>
              </div>
            </div>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>

      <a class="main-menu-open-button" href="#" data-behavior="open-mobile-menu"></a>
    </div>

  </div>
</div>


<body class="pytorch-body">

   

    

    <div class="table-of-contents-link-wrapper">
      <span>Table of Contents</span>
      <a href="#" class="toggle-table-of-contents" data-behavior="toggle-table-of-contents"></a>
    </div>

    <nav data-toggle="wy-nav-shift" class="pytorch-left-menu" id="pytorch-left-menu">
      <div class="pytorch-side-scroll">
        <div class="pytorch-menu pytorch-menu-vertical" data-spy="affix" role="navigation" aria-label="main navigation">
          <div class="pytorch-left-menu-search">
            

            
              
              
                <div class="version">
                  master (1.5.0 )
                </div>
              
            

            


  


<div role="search">
  <form id="rtd-search-form" class="wy-form" action="../../../../search.html" method="get">
    <input type="text" name="q" placeholder="Search Docs" />
    <input type="hidden" name="check_keywords" value="yes" />
    <input type="hidden" name="area" value="default" />
  </form>
</div>

            
          </div>

          
<div>
  <a style="color:#F05732" href="https://pytorch.org/docs/stable/_modules/torch/distributed/rpc/api.html">
    You are viewing unstable developer preview docs.
    Click here to view docs for latest stable release.
  </a>
</div>

            
            
              
            
            
              <p class="caption"><span class="caption-text">Notes</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/amp_examples.html">Automatic Mixed Precision examples</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/autograd.html">Autograd mechanics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/broadcasting.html">Broadcasting semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/cpu_threading_torchscript_inference.html">CPU threading and TorchScript inference</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/cuda.html">CUDA semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/ddp.html">Distributed Data Parallel</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/extending.html">Extending PyTorch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/faq.html">Frequently Asked Questions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/large_scale_deployments.html">Features for large-scale deployments</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/multiprocessing.html">Multiprocessing best practices</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/randomness.html">Reproducibility</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/serialization.html">Serialization semantics</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../notes/windows.html">Windows FAQ</a></li>
</ul>
<p class="caption"><span class="caption-text">Language Bindings</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/cppdocs/">C++ API</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../packages.html">Javadoc</a></li>
</ul>
<p class="caption"><span class="caption-text">Python API</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../torch.html">torch</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.html">torch.nn</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.functional.html">torch.nn.functional</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensors.html">torch.Tensor</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensor_attributes.html">Tensor Attributes</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensor_view.html">Tensor Views</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../autograd.html">torch.autograd</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../cuda.html">torch.cuda</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../amp.html">torch.cuda.amp</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../distributed.html">torch.distributed</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../distributions.html">torch.distributions</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../hub.html">torch.hub</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../jit.html">torch.jit</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../nn.init.html">torch.nn.init</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../onnx.html">torch.onnx</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../optim.html">torch.optim</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../quantization.html">Quantization</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../rpc/index.html">Distributed RPC Framework</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../random.html">torch.random</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../sparse.html">torch.sparse</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../storage.html">torch.Storage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../bottleneck.html">torch.utils.bottleneck</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../checkpoint.html">torch.utils.checkpoint</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../cpp_extension.html">torch.utils.cpp_extension</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../data.html">torch.utils.data</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../dlpack.html">torch.utils.dlpack</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../model_zoo.html">torch.utils.model_zoo</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../tensorboard.html">torch.utils.tensorboard</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../type_info.html">Type Info</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../named_tensor.html">Named Tensors</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../name_inference.html">Named Tensors operator coverage</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../__config__.html">torch.__config__</a></li>
</ul>
<p class="caption"><span class="caption-text">Libraries</span></p>
<ul>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/audio">torchaudio</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/text">torchtext</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/elastic/">TorchElastic</a></li>
<li class="toctree-l1"><a class="reference external" href="https://pytorch.org/serve">TorchServe</a></li>
<li class="toctree-l1"><a class="reference external" href="http://pytorch.org/xla/">PyTorch on XLA Devices</a></li>
</ul>
<p class="caption"><span class="caption-text">Community</span></p>
<ul>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/contribution_guide.html">PyTorch Contribution Guide</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/governance.html">PyTorch Governance</a></li>
<li class="toctree-l1"><a class="reference internal" href="../../../../community/persons_of_interest.html">PyTorch Governance | Persons of Interest</a></li>
</ul>

            
          

        </div>
      </div>
    </nav>

    <div class="pytorch-container">
      <div class="pytorch-page-level-bar" id="pytorch-page-level-bar">
        <div class="pytorch-breadcrumbs-wrapper">
          















<div role="navigation" aria-label="breadcrumbs navigation">

  <ul class="pytorch-breadcrumbs">
    
      <li>
        <a href="../../../../index.html">
          
            Docs
          
        </a> &gt;
      </li>

        
          <li><a href="../../../index.html">Module code</a> &gt;</li>
        
          <li><a href="../../../torch.html">torch</a> &gt;</li>
        
          <li><a href="../../distributed.html">torch.distributed</a> &gt;</li>
        
          <li><a href="../rpc.html">torch.distributed.rpc</a> &gt;</li>
        
      <li>torch.distributed.rpc.api</li>
    
    
      <li class="pytorch-breadcrumbs-aside">
        
      </li>
    
  </ul>

  
</div>
        </div>

        <div class="pytorch-shortcuts-wrapper" id="pytorch-shortcuts-wrapper">
          Shortcuts
        </div>
      </div>

      <section data-toggle="wy-nav-shift" id="pytorch-content-wrap" class="pytorch-content-wrap">
        <div class="pytorch-content-left">

        
          
          <div class="rst-content">
          
            <div role="main" class="main-content" itemscope="itemscope" itemtype="http://schema.org/Article">
             <article itemprop="articleBody" id="pytorch-article" class="pytorch-article">
              
  <h1>Source code for torch.distributed.rpc.api</h1><div class="highlight"><pre>
<span></span><span class="kn">import</span> <span class="nn">collections</span>
<span class="kn">import</span> <span class="nn">contextlib</span>
<span class="kn">import</span> <span class="nn">functools</span>
<span class="kn">import</span> <span class="nn">logging</span>
<span class="kn">import</span> <span class="nn">numbers</span>
<span class="kn">import</span> <span class="nn">sys</span>
<span class="kn">import</span> <span class="nn">threading</span>
<span class="kn">from</span> <span class="nn">datetime</span> <span class="kn">import</span> <span class="n">timedelta</span>

<span class="kn">import</span> <span class="nn">torch</span>
<span class="kn">import</span> <span class="nn">torch.distributed</span> <span class="k">as</span> <span class="nn">dist</span>

<span class="kn">from</span> <span class="nn">.</span> <span class="kn">import</span> <span class="p">(</span>
    <span class="n">RpcBackendOptions</span><span class="p">,</span>
    <span class="n">WorkerInfo</span><span class="p">,</span>
    <span class="n">_cleanup_python_rpc_handler</span><span class="p">,</span>
    <span class="n">_delete_all_user_rrefs</span><span class="p">,</span>
    <span class="n">_destroy_rref_context</span><span class="p">,</span>
    <span class="n">_get_current_rpc_agent</span><span class="p">,</span>
    <span class="n">_invoke_remote_builtin</span><span class="p">,</span>
    <span class="n">_invoke_remote_python_udf</span><span class="p">,</span>
    <span class="n">_invoke_remote_torchscript</span><span class="p">,</span>
    <span class="n">_invoke_rpc_builtin</span><span class="p">,</span>
    <span class="n">_invoke_rpc_python_udf</span><span class="p">,</span>
    <span class="n">_invoke_rpc_torchscript</span><span class="p">,</span>
    <span class="n">_is_current_rpc_agent_set</span><span class="p">,</span>
    <span class="n">_reset_current_rpc_agent</span><span class="p">,</span>
    <span class="n">_set_and_start_rpc_agent</span><span class="p">,</span>
    <span class="n">_set_rpc_timeout</span><span class="p">,</span>
    <span class="n">backend_registry</span><span class="p">,</span>
<span class="p">)</span>
<span class="kn">from</span> <span class="nn">.internal</span> <span class="kn">import</span> <span class="p">(</span>
    <span class="n">PythonUDF</span><span class="p">,</span>
    <span class="n">RPCExecMode</span><span class="p">,</span>
    <span class="n">_internal_rpc_pickler</span><span class="p">,</span>
    <span class="n">_start_record_function</span><span class="p">,</span>
<span class="p">)</span>


<span class="n">logger</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>


<span class="c1"># NB: Ignoring RRef leaks during shutdown. Without this, applications have to</span>
<span class="c1"># make sure there is no references to any RRef in the application code and</span>
<span class="c1"># Python GC has done its job to delete those RRefs. This is could result in bad</span>
<span class="c1"># debugging experiences especially when for large applications. Therefore, by</span>
<span class="c1"># default, we are going to ignore RRef leaks during shutdown. This is usually</span>
<span class="c1"># fine as shutdown means applications have done training and no longer care</span>
<span class="c1"># about states.</span>
<span class="c1">#</span>
<span class="c1"># To enable RRef leak checking, set this _ignore_rref_leak to False</span>
<span class="n">_ignore_rref_leak</span> <span class="o">=</span> <span class="kc">True</span>
<span class="n">_default_pickler</span> <span class="o">=</span> <span class="n">_internal_rpc_pickler</span>


<span class="nd">@contextlib</span><span class="o">.</span><span class="n">contextmanager</span>
<span class="k">def</span> <span class="nf">_use_rpc_pickler</span><span class="p">(</span><span class="n">rpc_pickler</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    rpc_pickler: (.internal._InternalRPCPickler) Overrides the default RPC pickler</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">global</span> <span class="n">_default_pickler</span>
    <span class="n">_default_pickler</span> <span class="o">=</span> <span class="n">rpc_pickler</span>
    <span class="k">try</span><span class="p">:</span>
        <span class="k">yield</span>
    <span class="k">finally</span><span class="p">:</span>
        <span class="n">_default_pickler</span> <span class="o">=</span> <span class="n">_internal_rpc_pickler</span>


<span class="k">def</span> <span class="nf">_require_initialized</span><span class="p">(</span><span class="n">func</span><span class="p">):</span>
    <span class="nd">@functools</span><span class="o">.</span><span class="n">wraps</span><span class="p">(</span><span class="n">func</span><span class="p">)</span>
    <span class="k">def</span> <span class="nf">wrapper</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="n">_is_current_rpc_agent_set</span><span class="p">():</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
                <span class="s2">&quot;RPC has not been initialized. Call &quot;</span>
                <span class="s2">&quot;torch.distributed.rpc.init_rpc first.&quot;</span>
            <span class="p">)</span>
        <span class="k">return</span> <span class="n">func</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>

    <span class="k">return</span> <span class="n">wrapper</span>


<span class="k">class</span> <span class="nc">WaitAllWorkersStates</span><span class="p">(</span><span class="nb">object</span><span class="p">):</span>
    <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span><span class="bp">self</span><span class="p">):</span>
        <span class="c1"># Each `intent_worker_names` is an empty set at beginning.</span>
        <span class="c1"># It&#39;s only used by leader worker. Leader worker is user-specified or</span>
        <span class="c1"># elected as the first worker in a sorted worker name list.</span>
        <span class="c1"># Whenever there is a worker showing shutdown intention to the leader, by</span>
        <span class="c1"># calling `_wait_all_workers()`, the leader adds this worker&#39;s name to the set.</span>
        <span class="c1"># The leader also adds itself&#39;s name to the set on calling</span>
        <span class="c1"># `_wait_all_workers()`. We need this because, we confine `_wait_all_workers()`</span>
        <span class="c1"># to be called only once, by examing if leader&#39;s name has been added to the set.</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">intent_worker_names</span> <span class="o">=</span> <span class="nb">set</span><span class="p">()</span>
        <span class="c1"># Once `intent_worker_names == _ALL_WORKER_NAMES`,</span>
        <span class="c1"># we flip `_SHUTDOWN_PROCEED_SIGNAL` on the leader, and leader will send RPCs</span>
        <span class="c1"># to follower workers to flip their `_SHUTDOWN_PROCEED_SIGNAL`s.</span>
        <span class="bp">self</span><span class="o">.</span><span class="n">proceed_signal</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Event</span><span class="p">()</span>


<span class="c1"># States used by `def _wait_all_workers()`.</span>
<span class="c1"># `_ALL_WORKER_NAMES` is initialized on initiaizing RPC layer.</span>
<span class="n">_ALL_WORKER_NAMES</span> <span class="o">=</span> <span class="kc">None</span>
<span class="n">_wait_all_workers_dict_lock</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Lock</span><span class="p">()</span>
<span class="n">_wait_all_workers_sequence_id</span> <span class="o">=</span> <span class="mi">0</span>
<span class="n">_wait_all_workers_sequence_id_to_states</span> <span class="o">=</span> <span class="n">collections</span><span class="o">.</span><span class="n">defaultdict</span><span class="p">(</span><span class="n">WaitAllWorkersStates</span><span class="p">)</span>


<span class="k">def</span> <span class="nf">_on_leader_follower_report_shutdown_intent</span><span class="p">(</span><span class="n">sequence_id</span><span class="p">,</span> <span class="n">worker_name</span><span class="p">):</span>
    <span class="k">assert</span> <span class="p">(</span>
        <span class="n">worker_name</span> <span class="ow">in</span> <span class="n">_ALL_WORKER_NAMES</span>
    <span class="p">),</span> <span class="s2">&quot;</span><span class="si">{worker_name}</span><span class="s2"> is not expected by leader.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">worker_name</span><span class="o">=</span><span class="n">worker_name</span><span class="p">)</span>
    <span class="n">intent_worker_names</span> <span class="o">=</span> <span class="n">_wait_all_workers_sequence_id_to_states</span><span class="p">[</span>
        <span class="n">sequence_id</span>
    <span class="p">]</span><span class="o">.</span><span class="n">intent_worker_names</span>
    <span class="k">assert</span> <span class="p">(</span>
        <span class="n">worker_name</span> <span class="ow">not</span> <span class="ow">in</span> <span class="n">intent_worker_names</span>
    <span class="p">),</span> <span class="s2">&quot;</span><span class="si">{worker_name}</span><span class="s2"> reported intent sequence id </span><span class="si">{sequence_id}</span><span class="s2"> twice. &quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
        <span class="n">worker_name</span><span class="o">=</span><span class="n">worker_name</span><span class="p">,</span> <span class="n">sequence_id</span><span class="o">=</span><span class="n">sequence_id</span>
    <span class="p">)</span>
    <span class="n">intent_worker_names</span><span class="o">.</span><span class="n">add</span><span class="p">(</span><span class="n">worker_name</span><span class="p">)</span>
    <span class="k">if</span> <span class="n">_ALL_WORKER_NAMES</span> <span class="o">==</span> <span class="n">intent_worker_names</span><span class="p">:</span>
        <span class="n">_set_proceed_shutdown_signal</span><span class="p">(</span><span class="n">sequence_id</span><span class="p">)</span>


<span class="k">def</span> <span class="nf">_set_proceed_shutdown_signal</span><span class="p">(</span><span class="n">sequence_id</span><span class="p">):</span>
    <span class="n">proceed_signal</span> <span class="o">=</span> <span class="n">_wait_all_workers_sequence_id_to_states</span><span class="p">[</span><span class="n">sequence_id</span><span class="p">]</span><span class="o">.</span><span class="n">proceed_signal</span>
    <span class="k">assert</span> <span class="p">(</span>
        <span class="ow">not</span> <span class="n">proceed_signal</span><span class="o">.</span><span class="n">is_set</span><span class="p">()</span>
    <span class="p">),</span> <span class="s2">&quot;Termination signal sequence id </span><span class="si">{}</span><span class="s2"> got set twice.&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
        <span class="n">sequence_id</span><span class="o">=</span><span class="n">sequence_id</span>
    <span class="p">)</span>
    <span class="n">proceed_signal</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>


<span class="nd">@_require_initialized</span>
<span class="k">def</span> <span class="nf">_wait_all_workers</span><span class="p">():</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Block until all local and remote RPC processes reach this method and wait</span>
<span class="sd">    for all outstanding work to complete. Every RPC process must call this</span>
<span class="sd">    method before exit to perform a graceful shutdown. This should be used to</span>
<span class="sd">    terminate the RPC framework, and there is no guarantee that the RPC</span>
<span class="sd">    framework will work after this method returns.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">assert</span> <span class="p">(</span>
        <span class="n">_ALL_WORKER_NAMES</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span>
    <span class="p">),</span> <span class="s2">&quot;`_ALL_WORKER_NAMES` is not initialized for `def _wait_all_workers`.&quot;</span>
    <span class="n">leader_worker_name</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="n">_ALL_WORKER_NAMES</span><span class="p">)[</span><span class="mi">0</span><span class="p">]</span>

    <span class="n">self_worker_name</span> <span class="o">=</span> <span class="n">_get_current_rpc_agent</span><span class="p">()</span><span class="o">.</span><span class="n">get_worker_info</span><span class="p">()</span><span class="o">.</span><span class="n">name</span>

    <span class="k">global</span> <span class="n">_wait_all_workers_sequence_id</span>
    <span class="k">with</span> <span class="n">_wait_all_workers_dict_lock</span><span class="p">:</span>
        <span class="n">sequence_id</span> <span class="o">=</span> <span class="n">_wait_all_workers_sequence_id</span>
        <span class="n">_wait_all_workers_sequence_id</span> <span class="o">+=</span> <span class="mi">1</span>

    <span class="n">is_leader_worker</span> <span class="o">=</span> <span class="n">leader_worker_name</span> <span class="o">==</span> <span class="n">self_worker_name</span>

    <span class="c1"># Phase 1: Followers send intents.</span>
    <span class="c1"># All followers report intents to the leader.</span>
    <span class="k">if</span> <span class="n">is_leader_worker</span><span class="p">:</span>
        <span class="n">_on_leader_follower_report_shutdown_intent</span><span class="p">(</span><span class="n">sequence_id</span><span class="p">,</span> <span class="n">self_worker_name</span><span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="n">rpc_sync</span><span class="p">(</span>
            <span class="n">leader_worker_name</span><span class="p">,</span>
            <span class="n">_on_leader_follower_report_shutdown_intent</span><span class="p">,</span>
            <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">sequence_id</span><span class="p">,</span> <span class="n">self_worker_name</span><span class="p">,),</span>
        <span class="p">)</span>

    <span class="n">proceed_signal</span> <span class="o">=</span> <span class="n">_wait_all_workers_sequence_id_to_states</span><span class="p">[</span>
        <span class="n">sequence_id</span>
    <span class="p">]</span><span class="o">.</span><span class="n">proceed_signal</span>
    <span class="n">proceed_signal</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>

    <span class="c1"># Phase 2: Leader asks followers to proceed.</span>
    <span class="c1"># Leader&#39;s signal is the first to be unblocked,</span>
    <span class="c1"># after receiving all followers&#39; intents.</span>
    <span class="k">if</span> <span class="n">is_leader_worker</span><span class="p">:</span>
        <span class="c1"># The leader sends out proceeed signals to all followers.</span>
        <span class="n">timeout</span> <span class="o">=</span> <span class="n">timedelta</span><span class="p">(</span><span class="n">seconds</span><span class="o">=</span><span class="mi">5</span><span class="p">)</span>
        <span class="n">_set_rpc_timeout</span><span class="p">(</span><span class="n">timeout</span><span class="p">)</span>
        <span class="n">worker_name_to_response_future_dict</span> <span class="o">=</span> <span class="nb">dict</span><span class="p">()</span>
        <span class="k">for</span> <span class="n">follower_worker_name</span> <span class="ow">in</span> <span class="n">_ALL_WORKER_NAMES</span> <span class="o">-</span> <span class="p">{</span><span class="n">leader_worker_name</span><span class="p">}:</span>
            <span class="n">fut</span> <span class="o">=</span> <span class="n">rpc_async</span><span class="p">(</span><span class="n">follower_worker_name</span><span class="p">,</span> <span class="n">_set_proceed_shutdown_signal</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="p">(</span><span class="n">sequence_id</span><span class="p">,))</span>
            <span class="n">worker_name_to_response_future_dict</span><span class="p">[</span><span class="n">follower_worker_name</span><span class="p">]</span> <span class="o">=</span> <span class="n">fut</span>
        <span class="k">for</span> <span class="n">follower_worker_name</span><span class="p">,</span> <span class="n">fut</span> <span class="ow">in</span> <span class="n">worker_name_to_response_future_dict</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
            <span class="k">try</span><span class="p">:</span>
                <span class="n">fut</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span>
            <span class="k">except</span> <span class="ne">RuntimeError</span> <span class="k">as</span> <span class="n">ex</span><span class="p">:</span>
                <span class="n">logger</span><span class="o">.</span><span class="n">error</span><span class="p">(</span>
                    <span class="s2">&quot;</span><span class="si">{worker_name}</span><span class="s2"> failed to respond to &#39;Shutdown Proceed.&#39; request in </span><span class="si">{timeout}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                        <span class="n">worker_name</span><span class="o">=</span><span class="n">follower_worker_name</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout</span>
                    <span class="p">)</span>
                <span class="p">)</span>


<div class="viewcode-block" id="shutdown"><a class="viewcode-back" href="../../../../rpc/rpc.html#torch.distributed.rpc.shutdown">[docs]</a><span class="nd">@_require_initialized</span>
<span class="k">def</span> <span class="nf">shutdown</span><span class="p">(</span><span class="n">graceful</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Perform a shutdown of the RPC agent, and then destroy the RPC agent. This</span>
<span class="sd">    stops the local agent from accepting outstanding requests, and shuts</span>
<span class="sd">    down the RPC framework by terminating all RPC threads. If ``graceful=True``,</span>
<span class="sd">    this will block until all local and remote RPC processes reach this method</span>
<span class="sd">    and wait for all outstanding work to complete. Otherwise, if</span>
<span class="sd">    ``graceful=False``, this is a local shutdown, and it does not wait for other</span>
<span class="sd">    RPC processes to reach this method.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        graceful (bool): Whether to do a graceful shutdown or not. If True,</span>
<span class="sd">                         this will 1) wait until there is no pending system</span>
<span class="sd">                         messages for ``UserRRefs`` and delete them; 2) block</span>
<span class="sd">                         until all local and remote RPC processes have reached</span>
<span class="sd">                         this method and wait for all outstanding work to</span>
<span class="sd">                         complete.</span>

<span class="sd">    Example::</span>
<span class="sd">        Make sure that ``MASTER_ADDRESS`` and ``MASTER_PORT`` are set properly</span>
<span class="sd">        on both workers. Refer to :meth:`~torch.distributed.init_process_group`</span>
<span class="sd">        API for more details. For example,</span>

<span class="sd">        &gt;&gt;&gt; export MASTER_ADDRESS=localhost</span>
<span class="sd">        &gt;&gt;&gt; export MASTER_port=5678</span>

<span class="sd">        Then run the following code in two different processes:</span>

<span class="sd">        &gt;&gt;&gt; # On worker 0:</span>
<span class="sd">        &gt;&gt;&gt; import torch</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker0&quot;, rank=0, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; # do some work</span>
<span class="sd">        &gt;&gt;&gt; result = rpc.rpc_sync(&quot;worker1&quot;, torch.add, args=(torch.ones(1), 1))</span>
<span class="sd">        &gt;&gt;&gt; # ready to shutdown</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">        &gt;&gt;&gt; # On worker 1:</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker1&quot;, rank=1, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; # wait for worker 0 to finish work, and then shutdown.</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="n">graceful</span><span class="p">:</span>
        <span class="n">_wait_all_workers</span><span class="p">()</span>
        <span class="n">_delete_all_user_rrefs</span><span class="p">()</span>
        <span class="n">_get_current_rpc_agent</span><span class="p">()</span><span class="o">.</span><span class="n">join</span><span class="p">()</span>
    <span class="k">try</span><span class="p">:</span>
        <span class="c1"># This raises a `TORCH_CHECK()` exception on RRef leak detected.</span>
        <span class="n">_destroy_rref_context</span><span class="p">(</span><span class="n">_ignore_rref_leak</span><span class="p">)</span>
    <span class="k">finally</span><span class="p">:</span>
        <span class="n">_get_current_rpc_agent</span><span class="p">()</span><span class="o">.</span><span class="n">shutdown</span><span class="p">()</span>
        <span class="c1"># clean up python rpc handler in shutdown(), see comments in</span>
        <span class="c1"># PythonRpcHandler::cleanup(), call it in python API because the</span>
        <span class="c1"># cleanup() function has python dependency, it assumes python</span>
        <span class="c1"># interpreter exists.</span>
        <span class="c1"># No matter if RRef leak exception is raised, this clean-up code</span>
        <span class="c1"># must run to avoid destruction segfault in Python 3.5.</span>
        <span class="n">_cleanup_python_rpc_handler</span><span class="p">()</span>
        <span class="n">_reset_current_rpc_agent</span><span class="p">()</span></div>


<span class="c1"># TODO: add a context manager to wrap _init_rpc_backend and shutdown</span>
<span class="k">def</span> <span class="nf">_init_rpc_backend</span><span class="p">(</span>
    <span class="n">backend</span><span class="o">=</span><span class="n">backend_registry</span><span class="o">.</span><span class="n">BackendType</span><span class="o">.</span><span class="n">PROCESS_GROUP</span><span class="p">,</span>
    <span class="n">store</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
    <span class="n">name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
    <span class="n">rank</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
    <span class="n">world_size</span><span class="o">=-</span><span class="mi">1</span><span class="p">,</span>
    <span class="n">rpc_backend_options</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<span class="p">):</span>

    <span class="k">if</span> <span class="n">sys</span><span class="o">.</span><span class="n">version_info</span> <span class="o">&lt;</span> <span class="p">(</span><span class="mi">3</span><span class="p">,</span> <span class="mi">0</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;RPC package does not support Python2.&quot;</span><span class="p">)</span>

    <span class="n">_validate_rpc_args</span><span class="p">(</span><span class="n">backend</span><span class="p">,</span> <span class="n">store</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">rank</span><span class="p">,</span> <span class="n">world_size</span><span class="p">,</span> <span class="n">rpc_backend_options</span><span class="p">)</span>

    <span class="k">if</span> <span class="n">_is_current_rpc_agent_set</span><span class="p">():</span>
        <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span><span class="s2">&quot;RPC is already initialized&quot;</span><span class="p">)</span>

    <span class="c1"># Initialize RPC.</span>
    <span class="n">rpc_agent</span> <span class="o">=</span> <span class="n">backend_registry</span><span class="o">.</span><span class="n">init_backend</span><span class="p">(</span>
        <span class="n">backend</span><span class="p">,</span>
        <span class="n">store</span><span class="o">=</span><span class="n">store</span><span class="p">,</span>
        <span class="n">name</span><span class="o">=</span><span class="n">name</span><span class="p">,</span>
        <span class="n">rank</span><span class="o">=</span><span class="n">rank</span><span class="p">,</span>
        <span class="n">world_size</span><span class="o">=</span><span class="n">world_size</span><span class="p">,</span>
        <span class="n">rpc_backend_options</span><span class="o">=</span><span class="n">rpc_backend_options</span><span class="p">,</span>
    <span class="p">)</span>

    <span class="n">worker_infos</span> <span class="o">=</span> <span class="n">rpc_agent</span><span class="o">.</span><span class="n">get_worker_infos</span><span class="p">()</span>
    <span class="k">global</span> <span class="n">_ALL_WORKER_NAMES</span>
    <span class="n">_ALL_WORKER_NAMES</span> <span class="o">=</span> <span class="p">{</span><span class="n">worker_info</span><span class="o">.</span><span class="n">name</span> <span class="k">for</span> <span class="n">worker_info</span> <span class="ow">in</span> <span class="n">worker_infos</span><span class="p">}</span>

    <span class="n">_set_and_start_rpc_agent</span><span class="p">(</span><span class="n">rpc_agent</span><span class="p">)</span>


<div class="viewcode-block" id="get_worker_info"><a class="viewcode-back" href="../../../../rpc/rpc.html#torch.distributed.rpc.get_worker_info">[docs]</a><span class="nd">@_require_initialized</span>
<span class="k">def</span> <span class="nf">get_worker_info</span><span class="p">(</span><span class="n">worker_name</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Get :class:`~torch.distributed.rpc.WorkerInfo` of a given worker name.</span>
<span class="sd">    Use this :class:`~torch.distributed.rpc.WorkerInfo` to avoid passing an</span>
<span class="sd">    expensive string on every invocation.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        worker_name (str): the string name of a worker. If ``None``, return the</span>
<span class="sd">                           the id of the current worker. (default ``None``)</span>

<span class="sd">    Returns:</span>
<span class="sd">        :class:`~torch.distributed.rpc.WorkerInfo` instance for the given</span>
<span class="sd">        ``worker_name`` or :class:`~torch.distributed.rpc.WorkerInfo` of the</span>
<span class="sd">        current worker if ``worker_name`` is ``None``.</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">if</span> <span class="n">worker_name</span><span class="p">:</span>
        <span class="k">return</span> <span class="n">_get_current_rpc_agent</span><span class="p">()</span><span class="o">.</span><span class="n">get_worker_info</span><span class="p">(</span><span class="n">worker_name</span><span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="k">return</span> <span class="n">_get_current_rpc_agent</span><span class="p">()</span><span class="o">.</span><span class="n">get_worker_info</span><span class="p">()</span></div>


<span class="k">def</span> <span class="nf">_to_worker_info</span><span class="p">(</span><span class="n">name_or_info</span><span class="p">):</span>
    <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name_or_info</span><span class="p">,</span> <span class="n">WorkerInfo</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">name_or_info</span>
    <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">name_or_info</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">get_worker_info</span><span class="p">(</span><span class="n">name_or_info</span><span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">&quot;Cannot get WorkerInfo from name </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">name_or_info</span><span class="p">))</span>


<span class="k">def</span> <span class="nf">_validate_rpc_args</span><span class="p">(</span><span class="n">backend</span><span class="p">,</span> <span class="n">store</span><span class="p">,</span> <span class="n">name</span><span class="p">,</span> <span class="n">rank</span><span class="p">,</span> <span class="n">world_size</span><span class="p">,</span> <span class="n">rpc_backend_options</span><span class="p">):</span>
    <span class="n">type_mapping</span> <span class="o">=</span> <span class="p">{</span>
        <span class="n">backend</span><span class="p">:</span> <span class="n">backend_registry</span><span class="o">.</span><span class="n">BackendType</span><span class="p">,</span>
        <span class="n">store</span><span class="p">:</span> <span class="n">dist</span><span class="o">.</span><span class="n">Store</span><span class="p">,</span>
        <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
        <span class="n">rank</span><span class="p">:</span> <span class="n">numbers</span><span class="o">.</span><span class="n">Integral</span><span class="p">,</span>
        <span class="n">world_size</span><span class="p">:</span> <span class="n">numbers</span><span class="o">.</span><span class="n">Integral</span><span class="p">,</span>
        <span class="n">rpc_backend_options</span><span class="p">:</span> <span class="n">RpcBackendOptions</span><span class="p">,</span>
    <span class="p">}</span>
    <span class="k">for</span> <span class="n">arg</span><span class="p">,</span> <span class="n">arg_type</span> <span class="ow">in</span> <span class="n">type_mapping</span><span class="o">.</span><span class="n">items</span><span class="p">():</span>
        <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">arg</span><span class="p">,</span> <span class="n">arg_type</span><span class="p">):</span>
            <span class="k">raise</span> <span class="ne">RuntimeError</span><span class="p">(</span>
                <span class="s2">&quot;Argument </span><span class="si">{}</span><span class="s2"> must be of type </span><span class="si">{}</span><span class="s2"> but got type </span><span class="si">{}</span><span class="s2">&quot;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                    <span class="n">arg</span><span class="p">,</span> <span class="n">arg_type</span><span class="p">,</span> <span class="nb">type</span><span class="p">(</span><span class="n">arg</span><span class="p">)</span>
                <span class="p">)</span>
            <span class="p">)</span>


<div class="viewcode-block" id="remote"><a class="viewcode-back" href="../../../../rpc/rpc.html#torch.distributed.rpc.remote">[docs]</a><span class="nd">@_require_initialized</span>
<span class="k">def</span> <span class="nf">remote</span><span class="p">(</span><span class="n">to</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Make a remote call to run ``func`` on worker ``to`` and return an</span>
<span class="sd">    :class:`~torch.distributed.rpc.RRef` to the result value immediately.</span>
<span class="sd">    Worker ``to`` will be the owner of the returned</span>
<span class="sd">    :class:`~torch.distributed.rpc.RRef`, and the worker calling ``remote`` is</span>
<span class="sd">    a user. The owner manages the global reference count of its</span>
<span class="sd">    :class:`~torch.distributed.rpc.RRef`, and the owner</span>
<span class="sd">    :class:`~torch.distributed.rpc.RRef` is only destructed when globally there</span>
<span class="sd">    are no living references to it.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        to (str or WorkerInfo): id or name of the destination worker.</span>
<span class="sd">        func (callable): a callable function, such as Python callables, builtin</span>
<span class="sd">                         operators (e.g. :meth:`~torch.add`) and annotated</span>
<span class="sd">                         TorchScript functions.</span>
<span class="sd">        args (tuple): the argument tuple for the ``func`` invocation.</span>
<span class="sd">        kwargs (dict): is a dictionary of keyword arguments for the ``func``</span>
<span class="sd">                       invocation.</span>

<span class="sd">    Returns:</span>
<span class="sd">        A user :class:`~torch.distributed.rpc.RRef` instance to the result</span>
<span class="sd">        value. Use the blocking API :meth:`torch.distributed.rpc.RRef.to_here`</span>
<span class="sd">        to retrieve the result value locally.</span>

<span class="sd">    .. warning ::</span>
<span class="sd">        Using GPU tensors as arguments or return values of ``func`` is not</span>
<span class="sd">        supported since we don&#39;t support sending GPU tensors over the wire. You</span>
<span class="sd">        need to explicitly copy GPU tensors to CPU before using them as</span>
<span class="sd">        arguments or return values of ``func``.</span>

<span class="sd">    .. warning ::</span>
<span class="sd">        The ``remote`` API does not copy storages of argument tensors until</span>
<span class="sd">        sending them over the wire, which could be done by a different thread</span>
<span class="sd">        depending on the RPC backend type. The caller should make sure that the</span>
<span class="sd">        contents of those tensors stay intact until the returned RRef is</span>
<span class="sd">        confirmed by the owner, which can be checked using the</span>
<span class="sd">        :meth:`torch.distributed.rpc.RRef.confirmed_by_owner` API.</span>

<span class="sd">    Example::</span>
<span class="sd">        Make sure that ``MASTER_ADDRESS`` and ``MASTER_PORT`` are set properly</span>
<span class="sd">        on both workers. Refer to :meth:`~torch.distributed.init_process_group`</span>
<span class="sd">        API for more details. For example,</span>

<span class="sd">        &gt;&gt;&gt; export MASTER_ADDRESS=localhost</span>
<span class="sd">        &gt;&gt;&gt; export MASTER_port=5678</span>

<span class="sd">        Then run the following code in two different processes:</span>

<span class="sd">        &gt;&gt;&gt; # On worker 0:</span>
<span class="sd">        &gt;&gt;&gt; import torch</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker0&quot;, rank=0, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; rref1 = rpc.remote(&quot;worker1&quot;, torch.add, args=(torch.ones(2), 3))</span>
<span class="sd">        &gt;&gt;&gt; rref2 = rpc.remote(&quot;worker1&quot;, torch.add, args=(torch.ones(2), 1))</span>
<span class="sd">        &gt;&gt;&gt; x = rref1.to_here() + rref2.to_here()</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">        &gt;&gt;&gt; # On worker 1:</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker1&quot;, rank=1, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">        Below is an example of running a TorchScript function using RPC.</span>

<span class="sd">        &gt;&gt;&gt; # On both workers:</span>
<span class="sd">        &gt;&gt;&gt; @torch.jit.script</span>
<span class="sd">        &gt;&gt;&gt; def my_script_add(t1, t2):</span>
<span class="sd">        &gt;&gt;&gt;    return torch.add(t1, t2)</span>

<span class="sd">        &gt;&gt;&gt; # On worker 0:</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker0&quot;, rank=0, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; rref = rpc.remote(&quot;worker1&quot;, my_script_add, args=(torch.ones(2), 3))</span>
<span class="sd">        &gt;&gt;&gt; rref.to_here()</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">        &gt;&gt;&gt; # On worker 1:</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker1&quot;, rank=1, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">qualified_name</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">_find_builtin</span><span class="p">(</span><span class="n">func</span><span class="p">)</span>
    <span class="n">dst_worker_info</span> <span class="o">=</span> <span class="n">_to_worker_info</span><span class="p">(</span><span class="n">to</span><span class="p">)</span>

    <span class="c1"># If profiling is enabled, kick off the timer and retrieve back a</span>
    <span class="c1"># RecordFunction instance.</span>
    <span class="n">rf</span> <span class="o">=</span> <span class="kc">None</span>
    <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">_profiler_enabled</span><span class="p">():</span>
        <span class="n">rf</span> <span class="o">=</span> <span class="n">_start_record_function</span><span class="p">(</span>
            <span class="n">RPCExecMode</span><span class="o">.</span><span class="n">REMOTE</span><span class="p">,</span>
            <span class="nb">str</span><span class="p">(</span><span class="n">qualified_name</span><span class="p">)</span> <span class="k">if</span> <span class="n">qualified_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">func</span><span class="o">.</span><span class="vm">__qualname__</span><span class="p">,</span>
            <span class="n">get_worker_info</span><span class="p">()</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
            <span class="n">dst_worker_info</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
        <span class="p">)</span>

    <span class="n">args</span> <span class="o">=</span> <span class="n">args</span> <span class="k">if</span> <span class="n">args</span> <span class="k">else</span> <span class="p">()</span>
    <span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span> <span class="k">if</span> <span class="n">kwargs</span> <span class="k">else</span> <span class="p">{}</span>

    <span class="k">if</span> <span class="n">qualified_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="k">return</span> <span class="n">_invoke_remote_builtin</span><span class="p">(</span><span class="n">dst_worker_info</span><span class="p">,</span> <span class="n">qualified_name</span><span class="p">,</span> <span class="n">rf</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
    <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">func</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">ScriptFunction</span><span class="p">):</span>
        <span class="k">return</span> <span class="n">_invoke_remote_torchscript</span><span class="p">(</span>
            <span class="n">dst_worker_info</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">_jit_internal</span><span class="o">.</span><span class="n">_qualified_name</span><span class="p">(</span><span class="n">func</span><span class="p">),</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span>
        <span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="p">(</span><span class="n">pickled_python_udf</span><span class="p">,</span> <span class="n">tensors</span><span class="p">)</span> <span class="o">=</span> <span class="n">_default_pickler</span><span class="o">.</span><span class="n">serialize</span><span class="p">(</span>
            <span class="n">PythonUDF</span><span class="p">(</span><span class="n">func</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">)</span>
        <span class="p">)</span>
        <span class="k">return</span> <span class="n">_invoke_remote_python_udf</span><span class="p">(</span><span class="n">dst_worker_info</span><span class="p">,</span> <span class="n">pickled_python_udf</span><span class="p">,</span> <span class="n">tensors</span><span class="p">,</span> <span class="n">rf</span><span class="p">)</span></div>


<span class="k">def</span> <span class="nf">_invoke_rpc</span><span class="p">(</span><span class="n">to</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="n">rpc_type</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="k">if</span> <span class="ow">not</span> <span class="n">callable</span><span class="p">(</span><span class="n">func</span><span class="p">):</span>
        <span class="k">raise</span> <span class="ne">TypeError</span><span class="p">(</span><span class="s2">&quot;function should be callable.&quot;</span><span class="p">)</span>

    <span class="n">qualified_name</span> <span class="o">=</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">_find_builtin</span><span class="p">(</span><span class="n">func</span><span class="p">)</span>
    <span class="n">dst_worker_info</span> <span class="o">=</span> <span class="n">_to_worker_info</span><span class="p">(</span><span class="n">to</span><span class="p">)</span>
    <span class="c1"># If profiling is enabled, kick off the timer and retrieve back a</span>
    <span class="c1"># RecordFunction instance.</span>
    <span class="n">rf</span> <span class="o">=</span> <span class="kc">None</span>
    <span class="k">if</span> <span class="n">torch</span><span class="o">.</span><span class="n">autograd</span><span class="o">.</span><span class="n">_profiler_enabled</span><span class="p">():</span>
        <span class="n">rf</span> <span class="o">=</span> <span class="n">_start_record_function</span><span class="p">(</span>
            <span class="n">rpc_type</span><span class="p">,</span>
            <span class="nb">str</span><span class="p">(</span><span class="n">qualified_name</span><span class="p">)</span> <span class="k">if</span> <span class="n">qualified_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">func</span><span class="o">.</span><span class="vm">__qualname__</span><span class="p">,</span>
            <span class="n">get_worker_info</span><span class="p">()</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
            <span class="n">dst_worker_info</span><span class="o">.</span><span class="n">name</span><span class="p">,</span>
        <span class="p">)</span>

    <span class="n">args</span> <span class="o">=</span> <span class="n">args</span> <span class="k">if</span> <span class="n">args</span> <span class="k">else</span> <span class="p">()</span>
    <span class="n">kwargs</span> <span class="o">=</span> <span class="n">kwargs</span> <span class="k">if</span> <span class="n">kwargs</span> <span class="k">else</span> <span class="p">{}</span>

    <span class="k">if</span> <span class="n">qualified_name</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
        <span class="n">fut</span> <span class="o">=</span> <span class="n">_invoke_rpc_builtin</span><span class="p">(</span><span class="n">dst_worker_info</span><span class="p">,</span> <span class="n">qualified_name</span><span class="p">,</span> <span class="n">rf</span><span class="p">,</span> <span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">)</span>
    <span class="k">elif</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">func</span><span class="p">,</span> <span class="n">torch</span><span class="o">.</span><span class="n">jit</span><span class="o">.</span><span class="n">ScriptFunction</span><span class="p">):</span>
        <span class="n">fut</span> <span class="o">=</span> <span class="n">_invoke_rpc_torchscript</span><span class="p">(</span><span class="n">dst_worker_info</span><span class="o">.</span><span class="n">name</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">)</span>
    <span class="k">else</span><span class="p">:</span>
        <span class="p">(</span><span class="n">pickled_python_udf</span><span class="p">,</span> <span class="n">tensors</span><span class="p">)</span> <span class="o">=</span> <span class="n">_default_pickler</span><span class="o">.</span><span class="n">serialize</span><span class="p">(</span>
            <span class="n">PythonUDF</span><span class="p">(</span><span class="n">func</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">)</span>
        <span class="p">)</span>
        <span class="n">fut</span> <span class="o">=</span> <span class="n">_invoke_rpc_python_udf</span><span class="p">(</span><span class="n">dst_worker_info</span><span class="p">,</span> <span class="n">pickled_python_udf</span><span class="p">,</span> <span class="n">tensors</span><span class="p">,</span> <span class="n">rf</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">fut</span>


<div class="viewcode-block" id="rpc_sync"><a class="viewcode-back" href="../../../../rpc/rpc.html#torch.distributed.rpc.rpc_sync">[docs]</a><span class="nd">@_require_initialized</span>
<span class="k">def</span> <span class="nf">rpc_sync</span><span class="p">(</span><span class="n">to</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Make a blocking RPC call to run function ``func`` on worker ``to``. RPC</span>
<span class="sd">    messages are sent and received in parallel to execution of Python code. This</span>
<span class="sd">    method is thread-safe.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        to (str or WorkerInfo): id or name of the destination worker.</span>
<span class="sd">        func (callable): a callable function, such as Python callables, builtin</span>
<span class="sd">                         operators (e.g. :meth:`~torch.add`) and annotated</span>
<span class="sd">                         TorchScript functions.</span>
<span class="sd">        args (tuple): the argument tuple for the ``func`` invocation.</span>
<span class="sd">        kwargs (dict): is a dictionary of keyword arguments for the ``func``</span>
<span class="sd">                       invocation.</span>

<span class="sd">    Returns:</span>
<span class="sd">        Returns the result of running ``func`` with ``args`` and ``kwargs``.</span>

<span class="sd">    .. warning ::</span>
<span class="sd">        Using GPU tensors as arguments or return values of ``func`` is not</span>
<span class="sd">        supported since we don&#39;t support sending GPU tensors over the wire. You</span>
<span class="sd">        need to explicitly copy GPU tensors to CPU before using them as</span>
<span class="sd">        arguments or return values of ``func``.</span>

<span class="sd">    Example::</span>
<span class="sd">        Make sure that ``MASTER_ADDRESS`` and ``MASTER_PORT`` are set properly</span>
<span class="sd">        on both workers. Refer to :meth:`~torch.distributed.init_process_group`</span>
<span class="sd">        API for more details. For example,</span>

<span class="sd">        &gt;&gt;&gt; export MASTER_ADDRESS=localhost</span>
<span class="sd">        &gt;&gt;&gt; export MASTER_port=5678</span>

<span class="sd">        Then run the following code in two different processes:</span>

<span class="sd">        &gt;&gt;&gt; # On worker 0:</span>
<span class="sd">        &gt;&gt;&gt; import torch</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker0&quot;, rank=0, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; ret = rpc.rpc_sync(&quot;worker1&quot;, torch.add, args=(torch.ones(2), 3))</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">        &gt;&gt;&gt; # On worker 1:</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker1&quot;, rank=1, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">        Below is an example of running a TorchScript function using RPC.</span>

<span class="sd">        &gt;&gt;&gt; # On both workers:</span>
<span class="sd">        &gt;&gt;&gt; @torch.jit.script</span>
<span class="sd">        &gt;&gt;&gt; def my_script_add(t1, t2):</span>
<span class="sd">        &gt;&gt;&gt;    return torch.add(t1, t2)</span>

<span class="sd">        &gt;&gt;&gt; # On worker 0:</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker0&quot;, rank=0, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; ret = rpc.rpc_sync(&quot;worker1&quot;, my_script_add, args=(torch.ones(2), 3))</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">        &gt;&gt;&gt; # On worker 1:</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker1&quot;, rank=1, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">    &quot;&quot;&quot;</span>
    <span class="n">fut</span> <span class="o">=</span> <span class="n">_invoke_rpc</span><span class="p">(</span><span class="n">to</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="n">RPCExecMode</span><span class="o">.</span><span class="n">SYNC</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">)</span>
    <span class="k">return</span> <span class="n">fut</span><span class="o">.</span><span class="n">wait</span><span class="p">()</span></div>


<div class="viewcode-block" id="rpc_async"><a class="viewcode-back" href="../../../../rpc/rpc.html#torch.distributed.rpc.rpc_async">[docs]</a><span class="nd">@_require_initialized</span>
<span class="k">def</span> <span class="nf">rpc_async</span><span class="p">(</span><span class="n">to</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">kwargs</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
    <span class="sa">r</span><span class="sd">&quot;&quot;&quot;</span>
<span class="sd">    Make a non-blocking RPC call to run function ``func`` on worker ``to``. RPC</span>
<span class="sd">    messages are sent and received in parallel to execution of Python code. This</span>
<span class="sd">    method is thread-safe. This method will immediately return a Future that can</span>
<span class="sd">    be awaited on.</span>

<span class="sd">    Arguments:</span>
<span class="sd">        to (str or WorkerInfo): id or name of the destination worker.</span>
<span class="sd">        func (callable): a callable function, such as Python callables, builtin</span>
<span class="sd">                         operators (e.g. :meth:`~torch.add`) and annotated</span>
<span class="sd">                         TorchScript functions.</span>
<span class="sd">        args (tuple): the argument tuple for the ``func`` invocation.</span>
<span class="sd">        kwargs (dict): is a dictionary of keyword arguments for the ``func``</span>
<span class="sd">                       invocation.</span>

<span class="sd">    Returns:</span>
<span class="sd">        Returns a Future object that can be waited</span>
<span class="sd">        on. When completed, the return value of ``func`` on ``args`` and</span>
<span class="sd">        ``kwargs`` can be retrieved from the Future object.</span>

<span class="sd">    .. warning ::</span>
<span class="sd">        Using GPU tensors as arguments or return values of ``func`` is not</span>
<span class="sd">        supported since we don&#39;t support sending GPU tensors over the wire. You</span>
<span class="sd">        need to explicitly copy GPU tensors to CPU before using them as</span>
<span class="sd">        arguments or return values of ``func``.</span>

<span class="sd">    .. warning ::</span>
<span class="sd">        The ``rpc_async`` API does not copy storages of argument tensors until</span>
<span class="sd">        sending them over the wire, which could be done by a different thread</span>
<span class="sd">        depending on the RPC backend type. The caller should make sure that the</span>
<span class="sd">        contents of those tensors stay intact until the returned Future</span>
<span class="sd">        completes.</span>

<span class="sd">    Example::</span>
<span class="sd">        Make sure that ``MASTER_ADDRESS`` and ``MASTER_PORT`` are set properly</span>
<span class="sd">        on both workers. Refer to :meth:`~torch.distributed.init_process_group`</span>
<span class="sd">        API for more details. For example,</span>

<span class="sd">        &gt;&gt;&gt; export MASTER_ADDRESS=localhost</span>
<span class="sd">        &gt;&gt;&gt; export MASTER_port=5678</span>

<span class="sd">        Then run the following code in two different processes:</span>

<span class="sd">        &gt;&gt;&gt; # On worker 0:</span>
<span class="sd">        &gt;&gt;&gt; import torch</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker0&quot;, rank=0, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; fut1 = rpc.rpc_async(&quot;worker1&quot;, torch.add, args=(torch.ones(2), 3))</span>
<span class="sd">        &gt;&gt;&gt; fut2 = rpc.rpc_async(&quot;worker1&quot;, min, args=(1, 2))</span>
<span class="sd">        &gt;&gt;&gt; result = fut1.wait() + fut2.wait()</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">        &gt;&gt;&gt; # On worker 1:</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker1&quot;, rank=1, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">        Below is an example of running a TorchScript function using RPC.</span>

<span class="sd">        &gt;&gt;&gt; # On both workers:</span>
<span class="sd">        &gt;&gt;&gt; @torch.jit.script</span>
<span class="sd">        &gt;&gt;&gt; def my_script_add(t1, t2):</span>
<span class="sd">        &gt;&gt;&gt;    return torch.add(t1, t2)</span>

<span class="sd">        &gt;&gt;&gt; # On worker 0:</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker0&quot;, rank=0, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; fut = rpc.rpc_async(&quot;worker1&quot;, my_script_add, args=(torch.ones(2), 3))</span>
<span class="sd">        &gt;&gt;&gt; ret = fut.wait()</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>

<span class="sd">        &gt;&gt;&gt; # On worker 1:</span>
<span class="sd">        &gt;&gt;&gt; import torch.distributed.rpc as rpc</span>
<span class="sd">        &gt;&gt;&gt; rpc.init_rpc(&quot;worker1&quot;, rank=1, world_size=2)</span>
<span class="sd">        &gt;&gt;&gt; rpc.shutdown()</span>
<span class="sd">    &quot;&quot;&quot;</span>
    <span class="k">return</span> <span class="n">_invoke_rpc</span><span class="p">(</span><span class="n">to</span><span class="p">,</span> <span class="n">func</span><span class="p">,</span> <span class="n">RPCExecMode</span><span class="o">.</span><span class="n">ASYNC</span><span class="p">,</span> <span class="n">args</span><span class="p">,</span> <span class="n">kwargs</span><span class="p">)</span></div>
</pre></div>

             </article>
             
            </div>
            <footer>
  

  

    <hr>

  

  <div role="contentinfo">
    <p>
        &copy; Copyright 2019, Torch Contributors.

    </p>
  </div>
    
      <div>
        Built with <a href="http://sphinx-doc.org/">Sphinx</a> using a <a href="https://github.com/rtfd/sphinx_rtd_theme">theme</a> provided by <a href="https://readthedocs.org">Read the Docs</a>.
      </div>
     

</footer>

          </div>
        </div>

        <div class="pytorch-content-right" id="pytorch-content-right">
          <div class="pytorch-right-menu" id="pytorch-right-menu">
            <div class="pytorch-side-scroll" id="pytorch-side-scroll-right">
              
            </div>
          </div>
        </div>
      </section>
    </div>

  


  

     
       <script type="text/javascript" id="documentation_options" data-url_root="../../../../" src="../../../../_static/documentation_options.js"></script>
         <script src="../../../../_static/jquery.js"></script>
         <script src="../../../../_static/underscore.js"></script>
         <script src="../../../../_static/doctools.js"></script>
         <script src="../../../../_static/language_data.js"></script>
     

  

  <script type="text/javascript" src="../../../../_static/js/vendor/popper.min.js"></script>
  <script type="text/javascript" src="../../../../_static/js/vendor/bootstrap.min.js"></script>
  <script src="https://cdnjs.cloudflare.com/ajax/libs/list.js/1.5.0/list.min.js"></script>
  <script type="text/javascript" src="../../../../_static/js/theme.js"></script>

  <script type="text/javascript">
      jQuery(function () {
          SphinxRtdTheme.Navigation.enable(true);
      });
  </script>
 
<script>
  (function(i,s,o,g,r,a,m){i['GoogleAnalyticsObject']=r;i[r]=i[r]||function(){
  (i[r].q=i[r].q||[]).push(arguments)},i[r].l=1*new Date();a=s.createElement(o),
  m=s.getElementsByTagName(o)[0];a.async=1;a.src=g;m.parentNode.insertBefore(a,m)
  })(window,document,'script','https://www.google-analytics.com/analytics.js','ga');

  ga('create', 'UA-90545585-1', 'auto');
  ga('send', 'pageview');

</script>

<script async src="https://www.googletagmanager.com/gtag/js?id=UA-117752657-2"></script>

<script>
  window.dataLayer = window.dataLayer || [];

  function gtag(){dataLayer.push(arguments);}

  gtag('js', new Date());
  gtag('config', 'UA-117752657-2');
</script>

<img height="1" width="1" style="border-style:none;" alt="" src="https://www.googleadservices.com/pagead/conversion/795629140/?label=txkmCPmdtosBENSssfsC&amp;guid=ON&amp;script=0"/>


  <!-- Begin Footer -->

  <div class="container-fluid docs-tutorials-resources" id="docs-tutorials-resources">
    <div class="container">
      <div class="row">
        <div class="col-md-4 text-center">
          <h2>Docs</h2>
          <p>Access comprehensive developer documentation for PyTorch</p>
          <a class="with-right-arrow" href="https://pytorch.org/docs/stable/index.html">View Docs</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Tutorials</h2>
          <p>Get in-depth tutorials for beginners and advanced developers</p>
          <a class="with-right-arrow" href="https://pytorch.org/tutorials">View Tutorials</a>
        </div>

        <div class="col-md-4 text-center">
          <h2>Resources</h2>
          <p>Find development resources and get your questions answered</p>
          <a class="with-right-arrow" href="https://pytorch.org/resources">View Resources</a>
        </div>
      </div>
    </div>
  </div>

  <footer class="site-footer">
    <div class="container footer-container">
      <div class="footer-logo-wrapper">
        <a href="https://pytorch.org/" class="footer-logo"></a>
      </div>

      <div class="footer-links-wrapper">
        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/">PyTorch</a></li>
            <li><a href="https://pytorch.org/get-started">Get Started</a></li>
            <li><a href="https://pytorch.org/features">Features</a></li>
            <li><a href="https://pytorch.org/ecosystem">Ecosystem</a></li>
            <li><a href="https://pytorch.org/blog/">Blog</a></li>
            <li><a href="https://github.com/pytorch/pytorch/blob/master/CONTRIBUTING.md">Contributing</a></li>
          </ul>
        </div>

        <div class="footer-links-col">
          <ul>
            <li class="list-title"><a href="https://pytorch.org/resources">Resources</a></li>
            <li><a href="https://pytorch.org/tutorials">Tutorials</a></li>
            <li><a href="https://pytorch.org/docs/stable/index.html">Docs</a></li>
            <li><a href="https://discuss.pytorch.org" target="_blank">Discuss</a></li>
            <li><a href="https://github.com/pytorch/pytorch/issues" target="_blank">Github Issues</a></li>
            <li><a href="https://pytorch.org/assets/brand-guidelines/PyTorch-Brand-Guidelines.pdf" target="_blank">Brand Guidelines</a></li>
          </ul>
        </div>

        <div class="footer-links-col follow-us-col">
          <ul>
            <li class="list-title">Stay Connected</li>
            <li>
              <div id="mc_embed_signup">
                <form
                  action="https://twitter.us14.list-manage.com/subscribe/post?u=75419c71fe0a935e53dfa4a3f&id=91d0dccd39"
                  method="post"
                  id="mc-embedded-subscribe-form"
                  name="mc-embedded-subscribe-form"
                  class="email-subscribe-form validate"
                  target="_blank"
                  novalidate>
                  <div id="mc_embed_signup_scroll" class="email-subscribe-form-fields-wrapper">
                    <div class="mc-field-group">
                      <label for="mce-EMAIL" style="display:none;">Email Address</label>
                      <input type="email" value="" name="EMAIL" class="required email" id="mce-EMAIL" placeholder="Email Address">
                    </div>

                    <div id="mce-responses" class="clear">
                      <div class="response" id="mce-error-response" style="display:none"></div>
                      <div class="response" id="mce-success-response" style="display:none"></div>
                    </div>    <!-- real people should not fill this in and expect good things - do not remove this or risk form bot signups-->

                    <div style="position: absolute; left: -5000px;" aria-hidden="true"><input type="text" name="b_75419c71fe0a935e53dfa4a3f_91d0dccd39" tabindex="-1" value=""></div>

                    <div class="clear">
                      <input type="submit" value="" name="subscribe" id="mc-embedded-subscribe" class="button email-subscribe-button">
                    </div>
                  </div>
                </form>
              </div>

            </li>
          </ul>

          <div class="footer-social-icons">
            <a href="https://www.facebook.com/pytorch" target="_blank" class="facebook"></a>
            <a href="https://twitter.com/pytorch" target="_blank" class="twitter"></a>
            <a href="https://www.youtube.com/pytorch" target="_blank" class="youtube"></a>
          </div>
        </div>
      </div>
    </div>
  </footer>

  <div class="cookie-banner-wrapper">
  <div class="container">
    <p class="gdpr-notice">To analyze traffic and optimize your experience, we serve cookies on this site. By clicking or navigating, you agree to allow our usage of cookies. As the current maintainers of this site, Facebook’s Cookies Policy applies. Learn more, including about available controls: <a href="https://www.facebook.com/policies/cookies/">Cookies Policy</a>.</p>
    <img class="close-button" src="../../../../_static/images/pytorch-x.svg">
  </div>
</div>

  <!-- End Footer -->

  <!-- Begin Mobile Menu -->

  <div class="mobile-main-menu">
    <div class="container-fluid">
      <div class="container">
        <div class="mobile-main-menu-header-container">
          <a class="header-logo" href="https://pytorch.org/" aria-label="PyTorch"></a>
          <a class="main-menu-close-button" href="#" data-behavior="close-mobile-menu"></a>
        </div>
      </div>
    </div>

    <div class="mobile-main-menu-links-container">
      <div class="main-menu">
        <ul>
          <li>
            <a href="https://pytorch.org/get-started">Get Started</a>
          </li>

          <li>
            <a href="https://pytorch.org/features">Features</a>
          </li>

          <li>
            <a href="https://pytorch.org/ecosystem">Ecosystem</a>
          </li>

          <li>
            <a href="https://pytorch.org/mobile">Mobile</a>
          </li>

          <li>
            <a href="https://pytorch.org/hub">PyTorch Hub</a>
          </li>

          <li>
            <a href="https://pytorch.org/blog/">Blog</a>
          </li>

          <li>
            <a href="https://pytorch.org/tutorials">Tutorials</a>
          </li>

          <li class="active">
            <a href="https://pytorch.org/docs/stable/index.html">Docs</a>
          </li>

          <li>
            <a href="https://pytorch.org/resources">Resources</a>
          </li>

          <li>
            <a href="https://github.com/pytorch/pytorch">Github</a>
          </li>
        </ul>
      </div>
    </div>
  </div>

  <!-- End Mobile Menu -->

  <script type="text/javascript" src="../../../../_static/js/vendor/anchor.min.js"></script>

  <script type="text/javascript">
    $(document).ready(function() {
      mobileMenu.bind();
      mobileTOC.bind();
      pytorchAnchors.bind();
      sideMenus.bind();
      scrollToAnchor.bind();
      highlightNavigation.bind();
      mainMenuDropdown.bind();
      filterTags.bind();

      // Remove any empty p tags that Sphinx adds
      $("[data-tags='null']").remove();

      // Add class to links that have code blocks, since we cannot create links in code blocks
      $("article.pytorch-article a span.pre").each(function(e) {
        $(this).closest("a").addClass("has-code");
      });
    })
  </script>
</body>
</html>